# -*- coding: utf-8 -*-
# @Time : 2022/8/5 下午2:52
# @Author : WangYunfei
# @FileName: demo6.py
# @Email: yunfei.wang@innvote.com
"""
6. 结合多线程采集以下网站2000条数据
地址 ： zb.yfb.qianlima.com/yfbsemsite/mesinfo/zbpglist
字段：时间、标题、地区、类型，交付：提供数据和代码截图即可。说明：禁止使用自动化等技术
"""
import hashlib
import time

import requests
from gevent.threadpool import ThreadPoolExecutor
from lxml import etree

from base import logger
from base.db.db_mongo import OpenMongoDB


class ZBMongoDB(OpenMongoDB):
    
    def insert_one(self, data: dict):
        """2.1 实现插入功能"""
        count = self.db.count_documents({"_id": data.get("_id")})
        if count == 0:
            # 替换mongodb的主键
            self.db.insert_one(data)
            logger.info("插入新的招标信息:{}".format(data))
        else:
            logger.warning("已经存在的招标信息:{}".format(data))

    @staticmethod
    def md5_str(data_str: str):
        md5 = hashlib.md5()
        md5.update(data_str.encode("utf-8"))
        return md5.hexdigest()


class ZB(object):
    def __init__(self, total=30, thread=10):
        """
            total: 爬取总数
            thread: 线程池线程数
        """
        self.url = "http://zb.yfb.qianlima.com/yfbsemsite/mesinfo/zbpglist"
        self.mongo = ZBMongoDB("zb", db="kaoshi")
        self.total = total
        self.thread = thread

    @staticmethod
    def md5_title(passwd: str):
        md5 = hashlib.md5()
        md5.update(passwd.encode("utf-8"))
        return md5.hexdigest()

    def get_html_data(self, form_data):
        """
        form_data: 参数
        """
        res = requests.post(self.url, data=form_data)
        result = etree.HTML(res.text).xpath('//*[@id="contentTable"]/tbody/tr')
        xpath_str = lambda x: str(x[0]).strip() if x else ""

        for item in result:
            title = xpath_str(item.xpath(".//a/text()"))
            zb_data = {
                "_id": self.md5_title(title),
                "时间": xpath_str(item.xpath("./td[1]/text()")),
                "标题": title,
                "地区": xpath_str(item.xpath("./td[2]/text()")),
                "类型": xpath_str(item.xpath("./td[3]/text()"))
            }
            self.mongo.insert_one(zb_data)

    def run(self):
        page_count = self.total // 30 + 1 if self.total % 30 != 0 else self.total // 30
        with ThreadPoolExecutor(self.thread) as f:
            for i in range(1, page_count+1):
                _form_data = {
                    "pageNo": i
                }
                f.submit(self.get_html_data, form_data=_form_data)


if __name__ == '__main__':
    # ZB().get_html_data(_form_data)
    start_time = time.time()
    ZB(2000).run()
    print(f"获取2000条数据并去重写入MongoDB用时{(time.time() - start_time) * 1000}毫秒")








